{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set tf 1.x for colab\n",
    "%tensorflow_version 1.x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Video \"What is TensorFlow\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%config IPCompleter.greedy=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.3.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "a = tf.placeholder(np.float32, (2, 2))\n",
    "b = tf.Variable(tf.ones((2, 2)))\n",
    "c = a @ b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"matmul:0\", shape=(2, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.,  2.],\n",
       "       [ 2.,  2.]], dtype=float32)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.run(tf.global_variables_initializer())\n",
    "s.run(c, feed_dict={a: np.ones((2, 2))})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Video \"Our first model in TensorFlow\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple optimization (with simple prints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "x = tf.get_variable(\"x\", shape=(), dtype=tf.float32, trainable=True)\n",
    "f = x ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(0.1)\n",
    "step = optimizer.minimize(f, var_list=[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'x:0' shape=() dtype=float32_ref>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.trainable_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.865988 1.17177\n",
      "-0.69279 0.749935\n",
      "-0.554232 0.479959\n",
      "-0.443386 0.307174\n",
      "-0.354709 0.196591\n",
      "-0.283767 0.125818\n",
      "-0.227014 0.0805237\n",
      "-0.181611 0.0515352\n",
      "-0.145289 0.0329825\n",
      "-0.116231 0.0211088\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as s:  # in this way session will be closed automatically\n",
    "    s.run(tf.global_variables_initializer())\n",
    "    for i in range(10):\n",
    "        _, curr_x, curr_f = s.run([step, x, f])\n",
    "        print(curr_x, curr_f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple optimization (with tf.Print)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "x = tf.get_variable(\"x\", shape=(), dtype=tf.float32)\n",
    "f = x ** 2\n",
    "f = tf.Print(f, [x, f], \"x, f:\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(0.1)\n",
    "step = optimizer.minimize(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as s:\n",
    "    s.run(tf.global_variables_initializer())\n",
    "    for i in range(10):\n",
    "        s.run([step, f])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prints to jupyter server stdout (not available in Coursera Notebooks):\n",
    "# 2018-07-21 18:01:27.308270: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-1.0670249][1.1385423]\n",
    "# 2018-07-21 18:01:27.308809: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.85361993][0.72866696]\n",
    "# 2018-07-21 18:01:27.309116: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.68289596][0.46634689]\n",
    "# 2018-07-21 18:01:27.309388: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.54631674][0.29846197]\n",
    "# 2018-07-21 18:01:27.309678: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.43705338][0.19101566]\n",
    "# 2018-07-21 18:01:27.309889: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.34964269][0.12225001]\n",
    "# 2018-07-21 18:01:27.310213: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.27971417][0.078240015]\n",
    "# 2018-07-21 18:01:27.310475: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.22377133][0.050073609]\n",
    "# 2018-07-21 18:01:27.310751: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.17901707][0.032047112]\n",
    "# 2018-07-21 18:01:27.310963: I tensorflow/core/kernels/logging_ops.cc:79] x, f:[-0.14321366][0.020510152]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple optimization (with TensorBoard logging)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "x = tf.get_variable(\"x\", shape=(), dtype=tf.float32)\n",
    "f = x ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(0.1)\n",
    "step = optimizer.minimize(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.summary.scalar('curr_x', x)\n",
    "tf.summary.scalar('curr_f', f)\n",
    "summaries = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.InteractiveSession()\n",
    "summary_writer = tf.summary.FileWriter(\"logs/1\", s.graph)\n",
    "s.run(tf.global_variables_initializer())\n",
    "for i in range(10):\n",
    "    _, curr_summaries = s.run([step, summaries])\n",
    "    summary_writer.add_summary(curr_summaries, i)\n",
    "    summary_writer.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run  `tensorboard --logdir=./logs` in bash"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is what you can see in your browser **(not available in Coursera Notebooks)**\n",
    "<img src=\"images/TensorBoard.jpg\" style=\"width:70%\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**If you're running on Google Colab you can still run TensorBoard!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !!! RUN THIS CELL ONLY ON GOOGLE COLAB !!!\n",
    "! wget https://raw.githubusercontent.com/hse-aml/intro-to-dl/master/setup_google_colab.py -O setup_google_colab.py\n",
    "import setup_google_colab\n",
    "\n",
    "# run tensorboard in background\n",
    "import os\n",
    "os.system(\"tensorboard --logdir=./logs --host 0.0.0.0 --port 6006 &\")\n",
    "\n",
    "# expose port and show the link\n",
    "setup_google_colab.expose_port_on_colab(6006)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training a linear model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 3) (1000, 1)\n",
      "[[ 0.09498027  0.48793618  0.39011257]]\n"
     ]
    }
   ],
   "source": [
    "# generate model data\n",
    "N = 1000\n",
    "D = 3\n",
    "x = np.random.random((N, D))\n",
    "w = np.random.random((D, 1))\n",
    "y = x @ w + np.random.randn(N, 1) * 0.20\n",
    "\n",
    "print(x.shape, y.shape)\n",
    "print(w.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 1) (?, 1) ()\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "features = tf.placeholder(tf.float32, shape=(None, D))\n",
    "target = tf.placeholder(tf.float32, shape=(None, 1))\n",
    "\n",
    "weights = tf.get_variable(\"weights\", shape=(D, 1), dtype=tf.float32)\n",
    "predictions = features @ weights\n",
    "\n",
    "loss = tf.reduce_mean((target - predictions) ** 2)\n",
    "\n",
    "print(target.shape, predictions.shape, loss.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(0.1)\n",
    "step = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.532865\n",
      "0.0458802\n",
      "0.0410158\n",
      "0.040087\n",
      "0.0399092\n",
      "0.0398751\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as s:\n",
    "    s.run(tf.global_variables_initializer())\n",
    "    for i in range(300):\n",
    "        _, curr_loss, curr_weights = s.run([step, loss, weights], \n",
    "                                           feed_dict={features: x, target: y})\n",
    "        if i % 50 == 0:\n",
    "            print(curr_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.11388827,  0.4882018 ,  0.36716884]], dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# found weights\n",
    "curr_weights.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.09498027,  0.48793618,  0.39011257]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# true weights\n",
    "w.T"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
